#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2018/8/17 11:44
# @Author  : youfeng
import json
import random
import time

import requests

from common.util import Util
from config import TEMP_REDIS_CONF
from dao.proxy_dao import ProxyDAO
from enum.proxy_key import ProxyKey
from ext.proxy_base import ProxyBase

__CLASS__ = "ProxyYizhou"
# 标注是否使用当前代理
__USED__ = True


class ProxyYizhou(ProxyBase):
    # 代理唯一标识
    __TAG__ = "yizhou"

    # 代理访问链接
    __YIZHOU_PROXY_API = 'http://too.ueuz.com/frontapi/public/http/get_ip/index?type=90&iptimelong=1&ipcount={}&protocol=0&areatype=1&area=&resulttype=txt&duplicate=1&separator=4&other=&show_city=0&show_carrier=0&show_expire=true&auth_key=e3378ee050d834f14875d1c91076cce0&timestamp=1536550561&sign=BADFEC85C587D372C47DA0E78E81B2A6'

    # 获得套餐信息链接
    __YIZHOU_PACKAGE_API = 'http://too.ueuz.com/frontapi/public/http/get_ip/todayRemain?package_id=90&auth_key=e3378ee050d834f14875d1c91076cce0'

    # 存储在redis中的key
    __PROXY_STORE_KEY = "ProxyYizhou:store:yizhou"

    # 最小生存时间 秒
    __PROXY_MIN_TTL = 30

    # 代理初始池大小 当代理池减小到0的时候 则进行一次初始化
    __PROXY_INIT_NUM = 50

    # 一天的秒数
    __DAILY_SECOND = 3600 * 24

    # 每日限定数目固定的
    __DAILY_LIMIT_PROXY_NUM = 100000

    # 每日重置时间
    __DAILY_LIMIT_TIME = "10:42:59"

    # __PROXY_NUM = 100
    def __init__(self, log):
        super(ProxyYizhou, self).__int__(log=log)
        self.__proxy_dao = ProxyDAO(TEMP_REDIS_CONF, log)

    def get_proxy_tag(self):
        return self.__TAG__

    # 获得当日到期时间
    def __get_next_expire_time(self):
        # 获得当前时间
        current_time = Util.get_current_time("%H:%M:%S")
        if current_time <= self.__DAILY_LIMIT_TIME:
            date = Util.get_current_date()
        else:
            date = Util.get_future_date(1)

        expire_time = "{} {}".format(date, self.__DAILY_LIMIT_TIME)
        return expire_time

    # 获得亿洲代理剩余次数和当前剩余时间
    def __get_package_info(self):
        package_day_limit = self.__DAILY_LIMIT_PROXY_NUM
        reset_timestamp = Util.time_to_timestamp(self.__get_next_expire_time())
        for _ in xrange(3):
            try:
                resp = requests.get(self.__YIZHOU_PACKAGE_API, timeout=10)
                if resp is None:
                    time.sleep(1)
                    continue

                if resp.status_code != 200:
                    continue

                json_data = json.loads(resp.text)
                if json_data is None:
                    continue

                state = json_data.get("state")
                if state != 0:
                    self.log.error("获取亿洲测试套餐状态失败: text = {}".format(resp.text))
                    continue

                data_dict = json_data.get("data")
                if not isinstance(data_dict, dict):
                    self.log.error("获取亿洲测试套餐data字段失败: text = {}".format(resp.text))
                    continue

                remain = data_dict.get("remain")
                if not isinstance(remain, int):
                    self.log.error("获取亿洲测试套餐remain字段失败: text = {}".format(resp.text))
                    continue

                if remain <= 0:
                    remain = 0

                return reset_timestamp, package_day_limit, remain
            except Exception as e:
                self.log.error("请求亿洲测试套餐信息失败:")
                self.log.exception(e)

        return -1, -1, -1

    # 获得当前可以获取代理个数
    def __get_current_proxy_num(self):
        expire_time, max_proxy_num, remain_num = self.__get_package_info()
        if remain_num <= 0 or max_proxy_num <= 0:
            self.log.error("当前亿洲测试已经没有额度了: limit_num = {} remain_num = {}".format(max_proxy_num, remain_num))
            return 0

        if not isinstance(expire_time, int):
            self.log.error("亿洲测试剩余时间异常: {}".format(expire_time))
            return 0

        # 获得还剩余多少时间
        current_time = int(time.time())
        last_second = expire_time - current_time
        if last_second <= 0:
            self.log.error("亿洲测试代理已过期: expire_time = {}".format(expire_time))
            return 0

        # 求出理论上应该剩余多少个数
        theory_remain_num = int((last_second / (self.__DAILY_SECOND * 1.0)) * max_proxy_num)

        self.log.info("亿洲测试代理: 过期时间 - {} 当前时间 - {} 剩余时间 - {}".format(expire_time, current_time, last_second))
        self.log.info("亿洲测试代理套餐数目: max_proxy_num = {}".format(max_proxy_num))
        self.log.info("亿洲测试代理理论剩余个数: theory_remain_num = {}".format(theory_remain_num))
        self.log.info("亿洲测试代理实际剩余个数: remain_num = {}".format(remain_num))

        # 用实际剩余个数减去理论剩余个数，则可以得出这次需要取出的个数
        if remain_num <= theory_remain_num:
            return 0

        return remain_num - theory_remain_num

    # 从亿洲获取代理 最大不能超过200
    def __get_proxy_list_from_yizhou(self, num):

        if num <= 0:
            return []

        times = int(num / 200)
        if times > 4:
            times = 4
            num = 200
        elif times < 1:
            times = 1
        else:
            num = 200

        # if num > 200:
        #     num = 200

        url = self.__YIZHOU_PROXY_API.format(num)
        proxy_list = []
        proxy_set = set()

        for __ in xrange(times):
            for _ in xrange(3):
                try:
                    r = requests.get(url, timeout=10)
                    if r is None or r.status_code != 200:
                        self.log.error("更新亿洲测试代理网络异常...")
                        time.sleep(random.randint(1, 3))
                        continue

                    if "{" in r.text or "}" in r.text:
                        self.log.error("亿洲测试理更新异常: text = {}".format(r.text))
                        time.sleep(random.randint(1, 2))
                        continue

                    line_list = r.text.strip().split('\n')
                    for line in line_list:
                        line = line.strip().strip("\r").strip("\n")
                        if len(line) <= 0:
                            continue

                        ip, time_str = line.split(",")
                        if ip in proxy_set:
                            continue

                        timestamp = Util.time_to_timestamp(time_str)
                        proxy = 'http://{}'.format(ip)
                        proxy_dict = {
                            ProxyKey.PROXY_KEY: proxy,
                            ProxyKey.EXPIRE_TIME_KEY: timestamp
                        }

                        proxy_list.append(proxy_dict)
                        proxy_set.add(ip)
                    break
                except Exception as e:
                    self.log.error('访问亿洲测试代理链接异常: ')
                    self.log.exception(e)
                    time.sleep(random.randint(1, 3))

        self.log.info('从亿洲测试代理官网更新代理成功: length = {}'.format(len(proxy_list)))
        return proxy_list

    # 合并并剔除不满足条件的代理
    def __merge_proxy_list(self, src1_proxy_list, src2_proxy_list):
        proxy_list = []
        proxy_set = set()

        # 获得当前时间
        current_time = int(time.time())

        def __merge(temp_proxy_list):
            for proxy_dict in temp_proxy_list:
                expire_time = proxy_dict.get(ProxyKey.EXPIRE_TIME_KEY)
                if expire_time is None:
                    self.log.error("当前亿洲测试代理没有到期时间: {}".format(proxy_dict))
                    continue

                if expire_time - current_time < self.__PROXY_MIN_TTL:
                    continue

                proxy = proxy_dict.get(ProxyKey.PROXY_KEY)
                if proxy is None:
                    self.log.error("当前亿洲测试代理没有代理信息: {}".format(proxy_dict))
                    continue

                if proxy in proxy_set:
                    continue

                proxy_set.add(proxy)
                proxy_list.append(proxy_dict)

        # 合并
        __merge(src1_proxy_list)
        __merge(src2_proxy_list)

        return proxy_list

    # 返回代理
    def get_proxy_list(self):

        # 获得redis中存储的代理
        redis_proxy_list = self.__proxy_dao.get_proxy_list_from_redis(self.__PROXY_STORE_KEY)
        self.log.info("当前redis中亿洲测试代理数目: num = {}".format(len(redis_proxy_list)))

        # 获得当前需要更新的代理个数
        can_get_num = self.__get_current_proxy_num()
        self.log.info("当前可以获取亿洲测试代理数目: num = {}".format(can_get_num))

        # 判断如果redis中并没有代理信息 则第一次提取多一点
        if len(redis_proxy_list) <= 0 and can_get_num < self.__PROXY_INIT_NUM:
            can_get_num = self.__PROXY_INIT_NUM
            self.log.info("当前需要获取的亿洲测试代理数目: num = {}".format(can_get_num))

        # 获得亿洲代理
        yizhou_proxy_list = self.__get_proxy_list_from_yizhou(can_get_num)
        self.log.info("通过接口获取亿洲测试代理数目: num = {}".format(len(yizhou_proxy_list)))

        # 合并结果
        merge_list = self.__merge_proxy_list(yizhou_proxy_list, redis_proxy_list)
        self.log.info("亿洲测试代理合并后数目: num = {}".format(len(merge_list)))

        # 先删除之前的结果 然后再存储
        self.__proxy_dao.delete_redis_proxy_list(self.__PROXY_STORE_KEY)

        # 存储到redis
        self.__proxy_dao.set_proxy_list_to_redis(self.__PROXY_STORE_KEY, merge_list)

        return merge_list


def main():
    # 标注类名
    from logger import AppLogger
    log = AppLogger('proxy_yizhou_test.log').get_logger()
    ProxyYizhou(log).get_proxy_list()


if __name__ == '__main__':
    main()
